Take-home Exercise 3

Getting Started

Background

FishEye International, a non-profit focused on countering illegal, unreported, and unregulated (IUU) fishing, has been given access to an international finance corporation’s database on fishing related companies. In the past, FishEye has determined that companies with anomalous structures are far more likely to be involved in IUU (or other “fishy” business). FishEye has transformed the database into a knowledge graph. It includes information about companies, owners, workers, and financial status. FishEye is aiming to use this graph to identify anomalies that could indicate a company is involved in IUU.

FishEye analysts have attempted to use traditional node-link visualizations and standard graph analyses, but these were found to be ineffective because the scale and detail in the data can obscure a business’s true structure. Can you help FishEye develop a new visual analytics approach to better understand fishing business anomalies?

Data Wrangling

Importing R pacakges

pacman::p_load(jsonlite,tidygraph,ggraph,visNetwork,graphlayouts,ggforce,skimr, 
               tidytext,tidyverse)

Importing JSON file by using jsonlite packages

mc3_data <- fromJSON("data/MC3.json")

Extracting edges

mc3_edges <- as_tibble(mc3_data$links) %>%
  distinct() %>%
  mutate(source = as.character(source),
         target = as.character(target),
         type = as.character(type)) %>%
  group_by(source, target, type) %>%
  summarise(weights=n()) %>%
  filter(source!=target) %>%
  ungroup()

Extracting nodes

mc3_nodes <- as_tibble(mc3_data$nodes) %>%
  mutate(country = as.character(country),
         id = as.character(id),
         product_services = as.character(product_services),
         revenue_omu = as.numeric(as.character(revenue_omu)),
         type = as.character(type)) %>%
  select(id, country, type, revenue_omu, product_services)

Exploring the edges data frame

Show the code

skim(mc3_edges)

Data summary

Name	mc3_edges
Number of rows	24036
Number of columns	4
_______________________
Column type frequency:
character	3
numeric	1
________________________
Group variables	None

Variable type: character

skim_variable	n_missing	complete_rate	min	max	empty	n_unique	whitespace
source	0	1	6	700	0	12856	0
target	0	1	6	28	0	21265	0
type	0	1	16	16	0	2	0

Variable type: numeric

skim_variable	n_missing	complete_rate	mean	sd	p0	p25	p50	p75	p100	hist
weights	0	1	1	0	1	1	1	1	1	▁▁▇▁▁

Edge table

Show the code

DT::datatable(mc3_edges)

Plotting by type

Show the code

ggplot(data = mc3_edges,
       aes(x = type)) +
  geom_bar()

Exploring the nodes data frame

Show the code

skim(mc3_nodes)

Data summary

Name	mc3_nodes
Number of rows	27622
Number of columns	5
_______________________
Column type frequency:
character	4
numeric	1
________________________
Group variables	None

Variable type: character

skim_variable	n_missing	complete_rate	min	max	empty	n_unique	whitespace
id	0	1	6	64	0	22929	0
country	0	1	2	15	0	100	0
type	0	1	7	16	0	3	0
product_services	0	1	4	1737	0	3244	0

Variable type: numeric

skim_variable	n_missing	complete_rate	mean	sd	p0	p25	p50	p75	p100	hist
revenue_omu	21515	0.22	1822155	18184433	3652.23	7676.36	16210.68	48327.66	310612303	▇▁▁▁▁

Node table

Show the code

DT::datatable(mc3_nodes)

Plotting by type

Show the code

ggplot(data = mc3_nodes,
       aes(x = type)) +
  geom_bar()

Visualisation and Analysis

Building network model

id1 <- mc3_edges %>%
  select(source) %>%
  rename(id = source)
id2 <- mc3_edges %>%
  select(target) %>%
  rename(id = target)
mc3_nodes1 <- rbind(id1, id2) %>%
  distinct() %>%
  left_join(mc3_nodes,
            unmatched = "drop")

mc3_graph <- tbl_graph(nodes = mc3_nodes1,
                       edges = mc3_edges,
                       directed = FALSE) %>%
  mutate(betweenness_centrality = centrality_betweenness(),
         closeness_centrality = centrality_closeness())

mc3_graph %>%
  filter(betweenness_centrality >= 100000) %>%
ggraph(layout = "fr") +
  geom_edge_link(aes(alpha=0.5)) +
  geom_node_point(aes(
    size = betweenness_centrality,
    colors = "lightblue",
    alpha = 0.5)) +
  scale_size_continuous(range=c(1,10))+
  theme_graph()

Text Sensing

Simple word count

mc3_nodes %>% 
    mutate(n_fish = str_count(product_services, "fish")) 

# A tibble: 27,622 × 6
   id                          country type  revenue_omu product_services n_fish
   <chr>                       <chr>   <chr>       <dbl> <chr>             <int>
 1 Jones LLC                   ZH      Comp…  310612303. Automobiles           0
 2 Coleman, Hall and Lopez     ZH      Comp…  162734684. Passenger cars,…      0
 3 Aqua Advancements Sashimi … Oceanus Comp…  115004667. Holding firm wh…      0
 4 Makumba Ltd. Liability Co   Utopor… Comp…   90986413. Car service, ca…      0
 5 Taylor, Taylor and Farrell  ZH      Comp…   81466667. Fully electric …      0
 6 Harmon, Edwards and Bates   ZH      Comp…   75070435. Discount superm…      0
 7 Punjab s Marine conservati… Riodel… Comp…   72167572. Beef, pork, chi…      0
 8 Assam   Limited Liability … Utopor… Comp…   72162317. Power and Gas s…      0
 9 Ianira Starfish Sagl Import Rio Is… Comp…   68832979. Light commercia…      0
10 Moran, Lewis and Jimenez    ZH      Comp…   65592906. Automobiles, tr…      0
# ℹ 27,612 more rows

Tokenisation

token_nodes <- mc3_nodes %>%
  unnest_tokens(word, 
                product_services)

token_nodes %>%
  count(word, sort = TRUE) %>%
  top_n(15) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(x = word, y = n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip() +
      labs(x = "Count",
      y = "Unique words",
      title = "Count of unique words found in product_services field")

Removing stopwords

stopwords_removed <- token_nodes %>% 
  anti_join(stop_words) %>%
  filter(word %in% c("seafood","fish","carp","catfish","herring","mackerel","pollock","salmon","shark","tuna"))
stopwords_removed %>%
  count(word, sort = TRUE) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(x = word, y = n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip() +
      labs(x = "Count",
      y = "Unique words",
      title = "Count of unique words found in product_services field")

Show the code

ggplot(data = stopwords_removed,
       aes(x = type)) +
  geom_bar()+
  geom_text(stat="count", 
      aes(label=paste0(..count..)),vjust=-1)+
  ylim(0,2250)

stopwords_removed[stopwords_removed$type=="Company Contacts",]

# A tibble: 1 × 5
  id                  country  type             revenue_omu word   
  <chr>               <chr>    <chr>                  <dbl> <chr>  
1 Fin Flair AG Marine Nalakond Company Contacts     157972. seafood

cleaned_edges <- mc3_edges %>%
  filter(source %in% stopwords_removed$id)

id3 <- cleaned_edges %>%
  select(source) %>%
  rename(id = source)
id4 <- cleaned_edges %>%
  select(target) %>%
  rename(id = target)
cleaned__nodes <- rbind(id3, id4) %>%
  distinct() %>%
  left_join(mc3_nodes,
            unmatched = "drop")

cleaned_graph <- tbl_graph(nodes = cleaned__nodes,
                       edges = cleaned_edges,
                       directed = FALSE) %>%
  mutate(betweenness_centrality = centrality_betweenness(),
         closeness_centrality = centrality_closeness())

cleaned_graph %>%
  filter(betweenness_centrality >= 10) %>%
ggraph(layout = "fr") +
  geom_edge_link(aes(alpha=0.5)) +
  geom_node_point(aes(
    size = betweenness_centrality,
    colors = "lightblue",
    alpha = 0.5)) +
  scale_size_continuous(range=c(1,10))+
  theme_graph()